Vehicle Detection

The goals / steps of this project are the following:

  • Perform a Histogram of Oriented Gradients (HOG) feature extraction on a labeled training set of images and train a classifier Linear SVM classifier
  • Optionally, you can also apply a color transform and append binned color features, as well as histograms of color, to your HOG feature vector.
  • Note: for those first two steps don't forget to normalize your features and randomize a selection for training and testing.
  • Implement a sliding-window technique and use your trained classifier to search for vehicles in images.
  • Run your pipeline on a video stream (start with the test_video.mp4 and later implement on full project_video.mp4) and create a heat map of recurring detections frame by frame to reject outliers and follow detected vehicles.
  • Estimate a bounding box for vehicles detected.

Preample

Import Statements

In [35]:
from multiprocessing import cpu_count, Pool, Manager, Process
from skimage.feature import hog
from sklearn.svm import LinearSVC, SVC
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, GridSearchCV
from scipy.ndimage.measurements import label
import matplotlib.pyplot as plt
from moviepy.editor import VideoFileClip
from IPython.display import HTML
import itertools
import numpy as np
import pickle
import cv2
import glob
import time
%matplotlib inline
In [36]:
manager = Manager()
car_images = manager.list()
noncar_images = manager.list()
taskbag = manager.Queue(2000)
outbag = manager.Queue(2000)

Dataset

In [37]:
car_image_paths = glob.glob('../vehicles/**/*.png')
noncar_image_paths = glob.glob('../non-vehicles/**/*.png')
print("Car Images: ", len(car_image_paths))
print("Non-Car Images: ", len(noncar_image_paths))
img = cv2.imread(car_image_paths[0])
print("Image size: ", img.shape)
Car Images:  8792
Non-Car Images:  8968
Image size:  (64, 64, 3)

Multiprocessing

In [ ]:
thread_pool = Pool(processes=cpu_count())
In [39]:
for path in car_image_paths:
    img = cv2.imread(path)
    car_images.append(img)
    
for path in noncar_image_paths:
    img = cv2.imread(path)
    noncar_images.append(img)
In [ ]:
def worker(i, tq, oq):
    while True:
        method, args = tq.get(True, None)
        ret = method(*args)
        oq.put(ret, True, None)
In [ ]:
workers = []
for i in range(cpu_count()):
    w = Process(target=worker, args=(i, taskbag, outbag))
    workers.append(w)
    w.start()

Feature Extraction

Histogram of Oriented Gradients (HOG) feature extraction

In [18]:
def get_hog_features(img, orient, pix_per_cell, cell_per_block, 
                        vis=False, feature_vec=True):
    # Call with two outputs if vis==True
    if vis == True:
        features, hog_image = hog(img, orientations=orient, 
                                  pixels_per_cell=(pix_per_cell, pix_per_cell),
                                  cells_per_block=(cell_per_block, cell_per_block),
                                  block_norm= 'L2-Hys',
                                  transform_sqrt=False, 
                                  visualise=vis, feature_vector=feature_vec)
        return features, hog_image
    # Otherwise call with one output
    else:      
        features = hog(img, orientations=orient, 
                       pixels_per_cell=(pix_per_cell, pix_per_cell),
                       cells_per_block=(cell_per_block, cell_per_block),
                       block_norm= 'L2-Hys',
                       transform_sqrt=False, 
                       visualise=vis, feature_vector=feature_vec)
        return features
In [19]:
car_img = cv2.imread(car_image_paths[0])
f, car_dst = get_hog_features(car_img[:,:,2], 9, 8, 8, vis=True, feature_vec=True)
ncar_img = cv2.imread(noncar_image_paths[0])
f, ncar_dst = get_hog_features(ncar_img[:,:,2], 9, 8, 8, vis=True, feature_vec=True)

# Visualize 
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(7,7))
f.subplots_adjust(hspace = .4, wspace=.2)
ax1.imshow(car_img)
ax1.set_title('Car Image', fontsize=16)
ax2.imshow(car_dst, cmap='gray')
ax2.set_title('HOG', fontsize=16)
ax3.imshow(ncar_img)
ax3.set_title('Non-Car Image', fontsize=16)
ax4.imshow(ncar_dst, cmap='gray')
ax4.set_title('HOG', fontsize=16)
Out[19]:
Text(0.5,1,'HOG')

Spatial Binning of Color feature extraction

In [20]:
def bin_spatial(img, size=(32, 32)):
    color1 = cv2.resize(img[:,:,0], size).ravel()
    color2 = cv2.resize(img[:,:,1], size).ravel()
    color3 = cv2.resize(img[:,:,2], size).ravel()
    return np.hstack((color1, color2, color3))
In [21]:
car_img = cv2.imread(car_image_paths[0])
car_dst = bin_spatial(car_img)
ncar_img = cv2.imread(noncar_image_paths[0])
ncar_dst = bin_spatial(ncar_img)

# Visualize 
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(7,7))
f.subplots_adjust(hspace = .4, wspace=.2)
ax1.imshow(car_img)
ax1.set_title('Car Image', fontsize=16)
ax2.plot(car_dst)
ax2.set_title('Bin Spatial', fontsize=16)
ax3.imshow(ncar_img)
ax3.set_title('Non-Car Image', fontsize=16)
ax4.plot(ncar_dst)
ax4.set_title('Bin Spatial', fontsize=16)
Out[21]:
Text(0.5,1,'Bin Spatial')

Color Histogram feature extraction

In [22]:
def color_hist(img, nbins=32, bins_range=(0, 256)):
    # Compute the histogram of the color channels separately
    channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
    channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
    channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
    # Concatenate the histograms into a single feature vector
    hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
    # Return the individual histograms, bin_centers and feature vector
    return hist_features
In [23]:
car_img = cv2.imread(car_image_paths[0])
car_dst = color_hist(car_img)
ncar_img = cv2.imread(noncar_image_paths[0])
ncar_dst = color_hist(ncar_img)

# Visualize 
f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(7,7))
f.subplots_adjust(hspace = .4, wspace=.2)
ax1.imshow(car_img)
ax1.set_title('Car Image', fontsize=16)
ax2.plot(car_dst)
ax2.set_title('Color Histogram', fontsize=16)
ax3.imshow(ncar_img)
ax3.set_title('Non-Car Image', fontsize=16)
ax4.plot(ncar_dst)
ax4.set_title('Color Histogram', fontsize=16)
Out[23]:
Text(0.5,1,'Color Histogram')

Feature Extraction parameter tuning

In [24]:
def color_conversion(image, cspace):
    # apply color conversion if other than 'BGR'
    if cspace != 'BGR':
        if cspace == 'RGB':
            feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        elif cspace == 'HSV':
            feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
        elif cspace == 'LUV':
            feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2LUV)
        elif cspace == 'HLS':
            feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2HLS)
        elif cspace == 'YUV':
            feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2YUV)
        elif cspace == 'YCrCb':
            feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2YCrCb)
    else: feature_image = np.copy(image)
    return feature_image
In [25]:
# Define a function to extract features from a list of images
def extract_features(imgs, cspace='YCrCb', orient=9, pix_per_cell=8, cell_per_block=2, hog_channel=0):
    # Create a list to append feature vectors to
    features = []
    # Iterate through the list of images
    for image in imgs:
        # Read in each one by one
        # image = cv2.imread(file)    
        feature_image = color_conversion(image, cspace)
        # Call get_hog_features() with vis=False, feature_vec=True
        channels = np.arange(feature_image.shape[2]) if hog_channel == 'ALL' else [hog_channel]
        local_features = []
        for channel in channels:
            local_features.append(get_hog_features(feature_image[:,:,channel], orient, pix_per_cell,
                                  cell_per_block, vis=False, feature_vec=True))
            # print('hog: ', local_features[-1].shape)
        local_features.append(bin_spatial(feature_image))
        # print('spatial: ', local_features[-1].shape)
        local_features.append(color_hist(feature_image))
        # print('color: ', local_features[-1].shape)
        features.append(np.hstack(local_features))
        # print('all:', features[-1].shape)
    # Return list of feature vectors
    # print('all:', features[-1].shape)
    return features
In [26]:
# Feature extraction parameters
def process_data(cspace='YCrCb', orient=9, pix_per_cell=8, cell_per_block=2, hog_channel=0):
    t = time.time()
    car_features = extract_features(car_images, cspace=cspace, orient=orient, 
                            pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, 
                            hog_channel=hog_channel)
    # print(car_features[0].shape)
    notcar_features = extract_features(noncar_images, cspace=cspace, orient=orient, 
                            pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, 
                            hog_channel=hog_channel)
    # print(notcar_features[0].shape)
    t2 = time.time()
    # print(round(t2-t, 2), 'Seconds to extract HOG features...')
    # Create an array stack of feature vectors
    X = np.vstack((car_features, notcar_features)).astype(np.float64)  
    # Fit a per-column scaler - this will be necessary if combining different types of features (HOG + color_hist/bin_spatial)
    X_scaler = StandardScaler().fit(X)
    # Apply the scaler to X
    scaled_X = X_scaler.transform(X)

    # Define the labels vector
    y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))

    # Split up data into randomized training and test sets
    rand_state = np.random.randint(0, 100)
    X_train, X_test, y_train, y_test = train_test_split(scaled_X, y, test_size=0.2, random_state=rand_state)
    #print('Feature vector length:', len(X_train[0]))
    return X_train, X_test, y_train, y_test, X_scaler, round(t2-t, 2)
In [27]:
def train_clf(X_train, y_train, kernel='rbf', C=40, gamma='auto'):
    clf = SVC(kernel=kernel, C=C, gamma=gamma)
    t = time.time()
    clf.fit(X_train, y_train)
    t2 = time.time()
    # print(round(t2 - t, 2), 'Seconds to train SVC...')
    return clf, round(t2 - t, 2)
In [28]:
def test_clf(clf, X_test, y_test):
    return round(clf.score(X_test, y_test), 4)
In [29]:
def tuner_mp(args):
    X_train, X_test, y_train, y_test, _, tm1 = process_data(*args)
    clf, tm2 = train_clf(X_train, y_train)
    acc = test_clf(clf, X_test, y_test)
    print("Accuracy:", acc, "Params:", args, "FE time:", tm1, "Train time:", tm2)
    return [acc, args, tm1, tm2]
In [44]:
def parameter_tuning(method):
    cspaces = ["RGB", "HSV", "LUV", "HLS", "YUV", "YCrCb"]
    orients = [9, 10, 11]
    ppcs = [8, 16]
    cpbs = [2]
    channels = [0, 1, 2, "ALL"]
    i = 0
    args = []
    
    for orient in orients:
        for ppc in ppcs:
            for cpb in cpbs:
                for cspace in cspaces:
                    for channel in channels:
                        i += 1
                        args.append([cspace, orient, ppc, cpb, channel])
    
    print("Exploring", i, "parameter configurations")
    results = thread_pool.map(tuner_mp, args)
    print("*** Finished exploration")
    for result in results:
        print("Accuracy:", result[0], "Params:", result[1], "FE time:", result[2], "Train time:", result[3])
In [45]:
parameter_tuning(tuner_mp)
Exploring 144 parameter configurations
*** Finished exploration
Accuracy: 0.9918 Params: ['RGB', 9, 8, 2, 0] FE time: 95.4 Train time: 531.99
Accuracy: 0.9913 Params: ['RGB', 9, 8, 2, 1] FE time: 88.52 Train time: 407.63
Accuracy: 0.9927 Params: ['RGB', 9, 8, 2, 2] FE time: 98.21 Train time: 442.89
Accuracy: 0.9896 Params: ['RGB', 9, 8, 2, 'ALL'] FE time: 266.88 Train time: 877.56
Accuracy: 0.9935 Params: ['HSV', 9, 8, 2, 0] FE time: 161.85 Train time: 323.56
Accuracy: 0.9932 Params: ['HSV', 9, 8, 2, 1] FE time: 110.02 Train time: 819.05
Accuracy: 0.9932 Params: ['HSV', 9, 8, 2, 2] FE time: 97.82 Train time: 340.26
Accuracy: 0.9935 Params: ['HSV', 9, 8, 2, 'ALL'] FE time: 248.62 Train time: 713.2
Accuracy: 0.9938 Params: ['LUV', 9, 8, 2, 0] FE time: 237.42 Train time: 448.61
Accuracy: 0.9924 Params: ['LUV', 9, 8, 2, 1] FE time: 141.79 Train time: 488.24
Accuracy: 0.9924 Params: ['LUV', 9, 8, 2, 2] FE time: 103.93 Train time: 782.97
Accuracy: 0.9941 Params: ['LUV', 9, 8, 2, 'ALL'] FE time: 240.31 Train time: 665.56
Accuracy: 0.9899 Params: ['HLS', 9, 8, 2, 0] FE time: 122.54 Train time: 376.16
Accuracy: 0.993 Params: ['HLS', 9, 8, 2, 1] FE time: 221.39 Train time: 428.62
Accuracy: 0.9938 Params: ['HLS', 9, 8, 2, 2] FE time: 147.61 Train time: 440.52
Accuracy: 0.9938 Params: ['HLS', 9, 8, 2, 'ALL'] FE time: 262.49 Train time: 621.38
Accuracy: 0.9935 Params: ['YUV', 9, 8, 2, 0] FE time: 95.72 Train time: 380.6
Accuracy: 0.9913 Params: ['YUV', 9, 8, 2, 1] FE time: 106.63 Train time: 559.03
Accuracy: 0.9904 Params: ['YUV', 9, 8, 2, 2] FE time: 104.4 Train time: 462.33
Accuracy: 0.9952 Params: ['YUV', 9, 8, 2, 'ALL'] FE time: 344.17 Train time: 828.04
Accuracy: 0.9941 Params: ['YCrCb', 9, 8, 2, 0] FE time: 96.11 Train time: 834.74
Accuracy: 0.9893 Params: ['YCrCb', 9, 8, 2, 1] FE time: 100.75 Train time: 389.7
Accuracy: 0.9879 Params: ['YCrCb', 9, 8, 2, 2] FE time: 93.23 Train time: 575.89
Accuracy: 0.9913 Params: ['YCrCb', 9, 8, 2, 'ALL'] FE time: 341.5 Train time: 671.39
Accuracy: 0.9918 Params: ['RGB', 9, 16, 2, 0] FE time: 129.71 Train time: 325.9
Accuracy: 0.9924 Params: ['RGB', 9, 16, 2, 1] FE time: 52.71 Train time: 291.26
Accuracy: 0.9921 Params: ['RGB', 9, 16, 2, 2] FE time: 210.27 Train time: 362.1
Accuracy: 0.9918 Params: ['RGB', 9, 16, 2, 'ALL'] FE time: 119.96 Train time: 318.95
Accuracy: 0.9918 Params: ['HSV', 9, 16, 2, 0] FE time: 60.15 Train time: 310.99
Accuracy: 0.9927 Params: ['HSV', 9, 16, 2, 1] FE time: 73.27 Train time: 355.21
Accuracy: 0.9952 Params: ['HSV', 9, 16, 2, 2] FE time: 54.63 Train time: 301.57
Accuracy: 0.9958 Params: ['HSV', 9, 16, 2, 'ALL'] FE time: 256.13 Train time: 375.77
Accuracy: 0.9893 Params: ['LUV', 9, 16, 2, 0] FE time: 61.82 Train time: 256.25
Accuracy: 0.9913 Params: ['LUV', 9, 16, 2, 1] FE time: 58.62 Train time: 238.52
Accuracy: 0.9935 Params: ['LUV', 9, 16, 2, 2] FE time: 63.54 Train time: 308.83
Accuracy: 0.9947 Params: ['LUV', 9, 16, 2, 'ALL'] FE time: 128.2 Train time: 363.04
Accuracy: 0.9918 Params: ['HLS', 9, 16, 2, 0] FE time: 65.09 Train time: 383.06
Accuracy: 0.9944 Params: ['HLS', 9, 16, 2, 1] FE time: 58.38 Train time: 304.95
Accuracy: 0.9913 Params: ['HLS', 9, 16, 2, 2] FE time: 58.44 Train time: 347.76
Accuracy: 0.9961 Params: ['HLS', 9, 16, 2, 'ALL'] FE time: 144.19 Train time: 418.22
Accuracy: 0.9896 Params: ['YUV', 9, 16, 2, 0] FE time: 68.76 Train time: 253.27
Accuracy: 0.9932 Params: ['YUV', 9, 16, 2, 1] FE time: 163.26 Train time: 305.94
Accuracy: 0.9924 Params: ['YUV', 9, 16, 2, 2] FE time: 54.28 Train time: 237.16
Accuracy: 0.9947 Params: ['YUV', 9, 16, 2, 'ALL'] FE time: 172.04 Train time: 338.49
Accuracy: 0.991 Params: ['YCrCb', 9, 16, 2, 0] FE time: 66.27 Train time: 262.24
Accuracy: 0.9918 Params: ['YCrCb', 9, 16, 2, 1] FE time: 100.28 Train time: 314.73
Accuracy: 0.9907 Params: ['YCrCb', 9, 16, 2, 2] FE time: 104.86 Train time: 318.13
Accuracy: 0.9947 Params: ['YCrCb', 9, 16, 2, 'ALL'] FE time: 163.2 Train time: 352.71
Accuracy: 0.9924 Params: ['RGB', 10, 8, 2, 0] FE time: 111.02 Train time: 504.06
Accuracy: 0.9935 Params: ['RGB', 10, 8, 2, 1] FE time: 118.32 Train time: 500.05
Accuracy: 0.9927 Params: ['RGB', 10, 8, 2, 2] FE time: 102.03 Train time: 365.78
Accuracy: 0.9924 Params: ['RGB', 10, 8, 2, 'ALL'] FE time: 246.08 Train time: 770.79
Accuracy: 0.9932 Params: ['HSV', 10, 8, 2, 0] FE time: 175.04 Train time: 398.84
Accuracy: 0.9921 Params: ['HSV', 10, 8, 2, 1] FE time: 112.56 Train time: 458.97
Accuracy: 0.9927 Params: ['HSV', 10, 8, 2, 2] FE time: 107.39 Train time: 382.6
Accuracy: 0.9932 Params: ['HSV', 10, 8, 2, 'ALL'] FE time: 347.22 Train time: 713.16
Accuracy: 0.9955 Params: ['LUV', 10, 8, 2, 0] FE time: 126.63 Train time: 512.38
Accuracy: 0.9932 Params: ['LUV', 10, 8, 2, 1] FE time: 170.74 Train time: 430.21
Accuracy: 0.9941 Params: ['LUV', 10, 8, 2, 2] FE time: 103.3 Train time: 620.04
Accuracy: 0.9952 Params: ['LUV', 10, 8, 2, 'ALL'] FE time: 268.31 Train time: 936.18
Accuracy: 0.991 Params: ['HLS', 10, 8, 2, 0] FE time: 113.42 Train time: 477.55
Accuracy: 0.9947 Params: ['HLS', 10, 8, 2, 1] FE time: 117.76 Train time: 482.51
Accuracy: 0.9913 Params: ['HLS', 10, 8, 2, 2] FE time: 134.05 Train time: 474.55
Accuracy: 0.9949 Params: ['HLS', 10, 8, 2, 'ALL'] FE time: 243.29 Train time: 623.98
Accuracy: 0.9949 Params: ['YUV', 10, 8, 2, 0] FE time: 119.16 Train time: 472.74
Accuracy: 0.9924 Params: ['YUV', 10, 8, 2, 1] FE time: 115.5 Train time: 527.03
Accuracy: 0.987 Params: ['YUV', 10, 8, 2, 2] FE time: 165.88 Train time: 623.2
Accuracy: 0.9924 Params: ['YUV', 10, 8, 2, 'ALL'] FE time: 268.37 Train time: 723.23
Accuracy: 0.993 Params: ['YCrCb', 10, 8, 2, 0] FE time: 102.85 Train time: 522.87
Accuracy: 0.9932 Params: ['YCrCb', 10, 8, 2, 1] FE time: 98.77 Train time: 490.73
Accuracy: 0.9901 Params: ['YCrCb', 10, 8, 2, 2] FE time: 109.82 Train time: 739.75
Accuracy: 0.9938 Params: ['YCrCb', 10, 8, 2, 'ALL'] FE time: 277.81 Train time: 779.74
Accuracy: 0.9916 Params: ['RGB', 10, 16, 2, 0] FE time: 73.22 Train time: 247.36
Accuracy: 0.9901 Params: ['RGB', 10, 16, 2, 1] FE time: 125.62 Train time: 278.84
Accuracy: 0.9938 Params: ['RGB', 10, 16, 2, 2] FE time: 68.71 Train time: 332.01
Accuracy: 0.9907 Params: ['RGB', 10, 16, 2, 'ALL'] FE time: 150.61 Train time: 362.63
Accuracy: 0.9916 Params: ['HSV', 10, 16, 2, 0] FE time: 66.82 Train time: 313.37
Accuracy: 0.9944 Params: ['HSV', 10, 16, 2, 1] FE time: 75.11 Train time: 348.24
Accuracy: 0.9935 Params: ['HSV', 10, 16, 2, 2] FE time: 106.63 Train time: 281.07
Accuracy: 0.9969 Params: ['HSV', 10, 16, 2, 'ALL'] FE time: 142.65 Train time: 260.55
Accuracy: 0.993 Params: ['LUV', 10, 16, 2, 0] FE time: 80.38 Train time: 306.76
Accuracy: 0.9885 Params: ['LUV', 10, 16, 2, 1] FE time: 74.91 Train time: 283.36
Accuracy: 0.9893 Params: ['LUV', 10, 16, 2, 2] FE time: 71.32 Train time: 345.18
Accuracy: 0.9935 Params: ['LUV', 10, 16, 2, 'ALL'] FE time: 192.57 Train time: 302.79
Accuracy: 0.9924 Params: ['HLS', 10, 16, 2, 0] FE time: 59.94 Train time: 288.99
Accuracy: 0.9941 Params: ['HLS', 10, 16, 2, 1] FE time: 73.43 Train time: 360.32
Accuracy: 0.9918 Params: ['HLS', 10, 16, 2, 2] FE time: 116.84 Train time: 305.16
Accuracy: 0.9932 Params: ['HLS', 10, 16, 2, 'ALL'] FE time: 123.62 Train time: 306.93
Accuracy: 0.9913 Params: ['YUV', 10, 16, 2, 0] FE time: 61.89 Train time: 338.56
Accuracy: 0.9913 Params: ['YUV', 10, 16, 2, 1] FE time: 66.53 Train time: 255.29
Accuracy: 0.9899 Params: ['YUV', 10, 16, 2, 2] FE time: 58.48 Train time: 279.39
Accuracy: 0.9972 Params: ['YUV', 10, 16, 2, 'ALL'] FE time: 109.9 Train time: 267.53
Accuracy: 0.9921 Params: ['YCrCb', 10, 16, 2, 0] FE time: 68.75 Train time: 344.82
Accuracy: 0.9901 Params: ['YCrCb', 10, 16, 2, 1] FE time: 65.98 Train time: 248.39
Accuracy: 0.9918 Params: ['YCrCb', 10, 16, 2, 2] FE time: 182.57 Train time: 392.27
Accuracy: 0.993 Params: ['YCrCb', 10, 16, 2, 'ALL'] FE time: 124.15 Train time: 378.48
Accuracy: 0.9918 Params: ['RGB', 11, 8, 2, 0] FE time: 107.21 Train time: 392.91
Accuracy: 0.9924 Params: ['RGB', 11, 8, 2, 1] FE time: 118.81 Train time: 540.62
Accuracy: 0.9924 Params: ['RGB', 11, 8, 2, 2] FE time: 126.39 Train time: 365.68
Accuracy: 0.9927 Params: ['RGB', 11, 8, 2, 'ALL'] FE time: 352.63 Train time: 799.6
Accuracy: 0.9907 Params: ['HSV', 11, 8, 2, 0] FE time: 104.95 Train time: 498.61
Accuracy: 0.9958 Params: ['HSV', 11, 8, 2, 1] FE time: 104.09 Train time: 410.69
Accuracy: 0.9935 Params: ['HSV', 11, 8, 2, 2] FE time: 275.58 Train time: 461.48
Accuracy: 0.9938 Params: ['HSV', 11, 8, 2, 'ALL'] FE time: 281.64 Train time: 685.14
Accuracy: 0.9958 Params: ['LUV', 11, 8, 2, 0] FE time: 104.52 Train time: 440.59
Accuracy: 0.9927 Params: ['LUV', 11, 8, 2, 1] FE time: 118.19 Train time: 643.63
Accuracy: 0.9918 Params: ['LUV', 11, 8, 2, 2] FE time: 129.01 Train time: 690.79
Accuracy: 0.9949 Params: ['LUV', 11, 8, 2, 'ALL'] FE time: 250.14 Train time: 741.89
Accuracy: 0.9896 Params: ['HLS', 11, 8, 2, 0] FE time: 104.64 Train time: 449.1
Accuracy: 0.9935 Params: ['HLS', 11, 8, 2, 1] FE time: 93.63 Train time: 316.61
Accuracy: 0.9924 Params: ['HLS', 11, 8, 2, 2] FE time: 226.87 Train time: 635.89
Accuracy: 0.9941 Params: ['HLS', 11, 8, 2, 'ALL'] FE time: 273.65 Train time: 611.21
Accuracy: 0.9944 Params: ['YUV', 11, 8, 2, 0] FE time: 107.55 Train time: 442.22
Accuracy: 0.9913 Params: ['YUV', 11, 8, 2, 1] FE time: 89.21 Train time: 505.32
Accuracy: 0.9885 Params: ['YUV', 11, 8, 2, 2] FE time: 99.65 Train time: 465.43
Accuracy: 0.9935 Params: ['YUV', 11, 8, 2, 'ALL'] FE time: 412.08 Train time: 912.42
Accuracy: 0.9918 Params: ['YCrCb', 11, 8, 2, 0] FE time: 106.61 Train time: 402.84
Accuracy: 0.989 Params: ['YCrCb', 11, 8, 2, 1] FE time: 95.97 Train time: 468.38
Accuracy: 0.9913 Params: ['YCrCb', 11, 8, 2, 2] FE time: 100.53 Train time: 593.34
Accuracy: 0.9927 Params: ['YCrCb', 11, 8, 2, 'ALL'] FE time: 175.19 Train time: 544.78
Accuracy: 0.9901 Params: ['RGB', 11, 16, 2, 0] FE time: 178.4 Train time: 343.43
Accuracy: 0.993 Params: ['RGB', 11, 16, 2, 1] FE time: 69.76 Train time: 275.52
Accuracy: 0.9916 Params: ['RGB', 11, 16, 2, 2] FE time: 162.43 Train time: 244.37
Accuracy: 0.9904 Params: ['RGB', 11, 16, 2, 'ALL'] FE time: 205.66 Train time: 338.38
Accuracy: 0.9924 Params: ['HSV', 11, 16, 2, 0] FE time: 60.69 Train time: 293.39
Accuracy: 0.9958 Params: ['HSV', 11, 16, 2, 1] FE time: 70.34 Train time: 356.35
Accuracy: 0.9952 Params: ['HSV', 11, 16, 2, 2] FE time: 76.67 Train time: 274.87
Accuracy: 0.9969 Params: ['HSV', 11, 16, 2, 'ALL'] FE time: 139.28 Train time: 317.89
Accuracy: 0.9904 Params: ['LUV', 11, 16, 2, 0] FE time: 133.25 Train time: 276.34
Accuracy: 0.9882 Params: ['LUV', 11, 16, 2, 1] FE time: 58.39 Train time: 258.05
Accuracy: 0.9904 Params: ['LUV', 11, 16, 2, 2] FE time: 87.83 Train time: 335.27
Accuracy: 0.9924 Params: ['LUV', 11, 16, 2, 'ALL'] FE time: 124.39 Train time: 374.17
Accuracy: 0.9913 Params: ['HLS', 11, 16, 2, 0] FE time: 65.35 Train time: 310.47
Accuracy: 0.9938 Params: ['HLS', 11, 16, 2, 1] FE time: 52.34 Train time: 295.39
Accuracy: 0.9952 Params: ['HLS', 11, 16, 2, 2] FE time: 60.89 Train time: 299.03
Accuracy: 0.9963 Params: ['HLS', 11, 16, 2, 'ALL'] FE time: 136.07 Train time: 339.64
Accuracy: 0.9904 Params: ['YUV', 11, 16, 2, 0] FE time: 58.49 Train time: 216.55
Accuracy: 0.991 Params: ['YUV', 11, 16, 2, 1] FE time: 41.62 Train time: 215.34
Accuracy: 0.9885 Params: ['YUV', 11, 16, 2, 2] FE time: 35.88 Train time: 194.6
Accuracy: 0.9972 Params: ['YUV', 11, 16, 2, 'ALL'] FE time: 64.13 Train time: 176.86
Accuracy: 0.9907 Params: ['YCrCb', 11, 16, 2, 0] FE time: 70.78 Train time: 293.5
Accuracy: 0.9882 Params: ['YCrCb', 11, 16, 2, 1] FE time: 56.22 Train time: 231.24
Accuracy: 0.9916 Params: ['YCrCb', 11, 16, 2, 2] FE time: 43.97 Train time: 221.66
Accuracy: 0.9963 Params: ['YCrCb', 11, 16, 2, 'ALL'] FE time: 81.49 Train time: 213.96

Classifier

Train a classifier

The parameter exploration above showed best accuracy (99.77%) with the following combination of parameters:

  • colorspace: HSV
  • channels: ALL
  • orientations: 10
  • pixels per cell: 16
  • cells per block: 2

Nonetheless, when performance is also considered, the best result is a few spots below no1, at the 2nd place with 99.72% accuracy and about half the execution time:

  • colorspace: YUV
  • channels: ALL
  • orientations: 11
  • pixels per cell: 16
  • cells per block: 2
In [46]:
colorspace = 'YUV'
orient = 11
pix_per_cell = 16
cell_per_block = 2
hog_channel = 'ALL' 
In [49]:
print("Extracting features")
X_train, X_test, y_train, y_test, X_scaler, tm1 = process_data(colorspace, orient, pix_per_cell, cell_per_block, hog_channel)
print("Features extracted in", tm1, "seconds")
print('Train set size:', len(X_train))
print('Test set size:', len(X_test))
print("Training classifier")
clf, tm2 = train_clf(X_train, y_train)
print("Trained in", tm2, "seconds")
acc = test_clf(clf, X_test, y_test)
print("Accuracy:", acc)
Extracting features
Features extracted in 54.98 seconds
Train set size: 14208
Test set size: 3552
Training classifier
Trained in 155.39 seconds
Accuracy: 0.9963

Find Cars

In [50]:
def find_cars(img, ystart, ystop, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block,  hog_channel=hog_channel, colorspace=colorspace,
              spatial_size=(32,32), hist_bins=32, hist_bins_range=(0,256), return_all=False):
    img_tosearch = img[ystart:ystop,:,:]
    ctrans_tosearch = color_conversion(img_tosearch, colorspace)
    if scale != 1:
        imshape = ctrans_tosearch.shape
        ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
    if hog_channel == 'ALL':
        ch1 = ctrans_tosearch[:,:,0]
        ch2 = ctrans_tosearch[:,:,1]
        ch3 = ctrans_tosearch[:,:,2]
    else:
        ch1 = ctrans_tosearch[:,:,hog_channel]
    
    # Define blocks and steps as above
    nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1
    nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1 
    nfeat_per_block = orient*cell_per_block**2
    
    # 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
    window = 64
    nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1
    cells_per_step = 2  # Instead of overlap, define how many cells to step
    nxsteps = (nxblocks - nblocks_per_window) // cells_per_step + 1
    nysteps = (nyblocks - nblocks_per_window) // cells_per_step + 1
    
    # Compute individual channel HOG features for the entire image
    hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
    if hog_channel == 'ALL':
        hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
        hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
    
    boxes = []
    for xb in range(nxsteps):
        for yb in range(nysteps):
            ypos = yb*cells_per_step
            xpos = xb*cells_per_step
            # Extract HOG for this patch
            features = []
            features.append(hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel())
            # print('hog: ', features[-1].shape)
            if hog_channel == 'ALL':
                features.append(hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel())
                # print('hog: ', features[-1].shape)
                features.append(hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel())
                # print('hog: ', features[-1].shape)

            xleft = xpos*pix_per_cell
            ytop = ypos*pix_per_cell

            # Extract the image patch
            subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64,64))
          
            # Get color features
            features.append(bin_spatial(subimg))
            # print('spatial: ', features[-1].shape)
            features.append(color_hist(subimg))
            # print('color: ', features[-1].shape)
            
            features_stacked = np.hstack(features)
            # Scale features and make a prediction
            test_features = X_scaler.transform([features_stacked])
            #test_features = X_scaler.transform(np.hstack((shape_feat, hist_feat)).reshape(1, -1))    
            test_prediction = clf.predict(test_features)
            
            if test_prediction == 1 or return_all:
                xbox_left = np.int(xleft*scale)
                ytop_draw = np.int(ytop*scale)
                win_draw = np.int(window*scale)
                boxes.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))
    return boxes
In [51]:
def draw_boxes(img, boxes, color=(0,0,255)):
    if color == '*':
        color = (np.random.randint(0,255), np.random.randint(0,255), np.random.randint(0,255))
    for box in boxes:
        cv2.rectangle(img, box[0], box[1], color, 6)
    return img
In [68]:
test_img = cv2.imread('./test_images/test1.jpg')
rgb_img = color_conversion(test_img, 'RGB')
ystart = 400
ystop = 656
scale = 1.5

t = time.time()
boxes = find_cars(test_img, ystart, ystop, scale, clf, X_scaler)
print(round(time.time() - t, 2), 'Seconds to find cars...')
t = time.time()
draw_img = draw_boxes(rgb_img, boxes)
print(round(time.time() - t, 2), 'Seconds to draw boxes...')

print(len(boxes), 'boxes found in image')
plt.figure(figsize = (12,7))
plt.imshow(draw_img, aspect='auto')
1.25 Seconds to find cars...
0.0 Seconds to draw boxes...
5 boxes found in image
Out[68]:
<matplotlib.image.AxesImage at 0x7fb3249958d0>

Search & Bound

Experiment with different search areas and sizes

Scale 1.0

In [69]:
rgb_img = color_conversion(test_img, 'RGB')
scale = 1.0
boxes = []

t = time.time()
boxes.extend(find_cars(test_img, 400, 464, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')
t = time.time()
boxes.extend(find_cars(test_img, 416, 480, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')

draw_img = draw_boxes(rgb_img, boxes, color='*')

plt.figure(figsize=(12,7))
plt.imshow(draw_img)
print(len(boxes), 'boxes found in image')
0.52 Seconds to find cars...
0.49 Seconds to find cars...
78 boxes found in image

scale 1.5

In [70]:
rgb_img = color_conversion(test_img, 'RGB')
scale = 1.5
boxes = []

t = time.time()
boxes.extend(find_cars(test_img, 400, 496, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')
t = time.time()
boxes.extend(find_cars(test_img, 432, 528, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')

draw_img = draw_boxes(rgb_img, boxes, color='*')

plt.figure(figsize=(12,7))
plt.imshow(draw_img)
print(len(boxes), 'boxes found in image')
0.34 Seconds to find cars...
0.34 Seconds to find cars...
50 boxes found in image

scale 2.0

In [71]:
rgb_img = color_conversion(test_img, 'RGB')
scale = 2.0
boxes = []

t = time.time()
boxes.extend(find_cars(test_img, 400, 528, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')
t = time.time()
boxes.extend(find_cars(test_img, 432, 560, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')

draw_img = draw_boxes(rgb_img, boxes, color='*')

plt.figure(figsize=(12,7))
plt.imshow(draw_img)
print(len(boxes), 'boxes found in image')
0.26 Seconds to find cars...
0.27 Seconds to find cars...
38 boxes found in image

scale 3.0

In [72]:
rgb_img = color_conversion(test_img, 'RGB')
scale = 3.0
boxes = []

t = time.time()
boxes.extend(find_cars(test_img, 400, 596, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')
t = time.time()
boxes.extend(find_cars(test_img, 464, 660, scale, clf, X_scaler, return_all=True))
print(round(time.time() - t, 2), 'Seconds to find cars...')

draw_img = draw_boxes(rgb_img, boxes, color='*')

plt.figure(figsize=(12,7))
plt.imshow(draw_img)
print(len(boxes), 'boxes found in image')
0.19 Seconds to find cars...
0.16 Seconds to find cars...
24 boxes found in image

Scale combination

In [77]:
rgb_img = color_conversion(test_img, 'RGB')

boxes = []
t = time.time()
scale = 3.0
boxes.extend(find_cars(test_img, 400, 596, scale, clf, X_scaler))
boxes.extend(find_cars(test_img, 464, 660, scale, clf, X_scaler))
scale = 2.0
boxes.extend(find_cars(test_img, 400, 528, scale, clf, X_scaler))
boxes.extend(find_cars(test_img, 432, 560, scale, clf, X_scaler))
scale = 1.5
boxes.extend(find_cars(test_img, 400, 496, scale, clf, X_scaler))
boxes.extend(find_cars(test_img, 432, 528, scale, clf, X_scaler))
scale = 1.0
boxes.extend(find_cars(test_img, 400, 464, scale, clf, X_scaler))
boxes.extend(find_cars(test_img, 416, 480, scale, clf, X_scaler))
print(round(time.time() - t, 2), 'Seconds to find cars...')

rgb_img = draw_boxes(rgb_img, boxes)

plt.figure(figsize=(12,7))
plt.imshow(rgb_img)
print(len(boxes), 'boxes found in image')
2.54 Seconds to find cars...
23 boxes found in image

Heatmap

In [78]:
def add_heat(heatmap, bbox_list):
    # Iterate through list of bboxes
    for box in bbox_list:
        # Add += 1 for all pixels inside each bbox
        # Assuming each "box" takes the form ((x1, y1), (x2, y2))
        heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1

    # Return updated heatmap
    return heatmap# Iterate through list of bboxes
In [79]:
heatmap_img = np.zeros_like(test_img[:,:,0])
heatmap_img = add_heat(heatmap_img, boxes)
plt.figure(figsize=(12,7))
plt.imshow(heatmap_img, cmap='hot')
Out[79]:
<matplotlib.image.AxesImage at 0x7fb33c63ea58>
In [80]:
def apply_threshold(heatmap, threshold):
    # Zero out pixels below the threshold
    heatmap[heatmap <= threshold] = 0
    # Return thresholded map
    return heatmap
In [81]:
heatmap_img = apply_threshold(heatmap_img, 1)
plt.figure(figsize=(12,7))
plt.imshow(heatmap_img, cmap='hot')
Out[81]:
<matplotlib.image.AxesImage at 0x7fb32492a6d8>

Label heatmap

In [82]:
labels = label(heatmap_img)
plt.figure(figsize=(12,7))
plt.imshow(labels[0], cmap='gray')
print(labels[1], 'cars found')
2 cars found

Draw bounding boxes

In [83]:
def draw_labeled_bboxes(img, labels):
    # Iterate through all detected cars
    boxes = []
    for car_number in range(1, labels[1]+1):
        # Find pixels with each car_number label value
        nonzero = (labels[0] == car_number).nonzero()
        # Identify x and y values of those pixels
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Define a bounding box based on min/max x and y
        boxes.append(((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy))))
        # Draw the box on the image
        cv2.rectangle(img, boxes[-1][0], boxes[-1][1], (0,0,255), 6)
    # Return the image
    return img, boxes
In [84]:
rgb_img = color_conversion(test_img, 'RGB')
# Draw bounding boxes on a copy of the image
draw_img, boxes = draw_labeled_bboxes(rgb_img, labels)
# Display the image
plt.figure(figsize=(12,7))
plt.imshow(draw_img)
Out[84]:
<matplotlib.image.AxesImage at 0x7fb3248a6630>

Video Pipeline

In [85]:
def pipeline(img):
    colorspace = 'YUV'
    orient = 11
    pix_per_cell = 16
    cell_per_block = 2
    hog_channel = 'ALL' 
    
    boxes = []
    
    scale = 3.0
    boxes.extend(find_cars(img, 400, 596, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 464, 660, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block,  hog_channel=hog_channel, colorspace=colorspace))
    scale = 2.0
    boxes.extend(find_cars(img, 400, 528, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 432, 560, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block,  hog_channel=hog_channel, colorspace=colorspace))
    scale = 1.5
    boxes.extend(find_cars(img, 400, 496, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 432, 528, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block,  hog_channel=hog_channel, colorspace=colorspace))
    scale = 1.0
    boxes.extend(find_cars(img, 400, 464, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 416, 480, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    
    heatmap_img = np.zeros_like(img[:,:,0])
    heatmap_img = add_heat(heatmap_img, boxes)
    heatmap_img = apply_threshold(heatmap_img, 1)
    labels = label(heatmap_img)
    rgb_img = color_conversion(img, 'RGB')
    draw_img, rects = draw_labeled_bboxes(rgb_img, labels)
    return draw_img

def video_preproc(img):
    img2 = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
    return pipeline(img2)

Execute on test images

In [86]:
test_images = glob.glob('./test_images/test*.jpg')

fig, axs = plt.subplots(3, 2, figsize=(16,14))
fig.subplots_adjust(hspace = .004, wspace=.002)
axs = axs.ravel()

for i, im in enumerate(test_images):
    axs[i].imshow(pipeline(cv2.imread(im)))
    axs[i].axis('off')

Execute on test video

In [87]:
clip_test = VideoFileClip('test_video.mp4')
clip_test_out = clip_test.fl_image(video_preproc)
%time clip_test_out.write_videofile('test_video_out.mp4', audio=False)
[MoviePy] >>>> Building video test_video_out.mp4
[MoviePy] Writing video test_video_out.mp4
 97%|█████████▋| 38/39 [01:44<00:02,  2.75s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: test_video_out.mp4 

CPU times: user 6min 45s, sys: 6.58 s, total: 6min 52s
Wall time: 1min 45s
In [88]:
HTML("""
<video width="640" height="480" controls>
  <source src="{0}">
</video>
""".format('test_video_out.mp4'))
Out[88]:

Implement a detection cache

In [89]:
class DetectionCache():
    
    def __init__(self, size = 15):
        # history of rectangles previous n frames
        self.boxes = []
        self.size = size
        
    def add_boxes(self, boxes):
        if len(boxes) == 0:
            return
        self.boxes.append(boxes)
        if len(self.boxes) > self.size:
            # throw out oldest rectangle set(s)
            self.boxes = self.boxes[len(self.boxes)-self.size:]

Pipeline with cache

In [90]:
def pipelineCache(img):
    colorspace = 'YUV'
    orient = 11
    pix_per_cell = 16
    cell_per_block = 2
    hog_channel = 'ALL' 
    
    boxes = []
    
    scale = 3.0
    boxes.extend(find_cars(img, 400, 596, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 464, 660, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block,  hog_channel=hog_channel, colorspace=colorspace))
    scale = 2.0
    boxes.extend(find_cars(img, 400, 528, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 432, 560, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block,  hog_channel=hog_channel, colorspace=colorspace))
    scale = 1.5
    boxes.extend(find_cars(img, 400, 496, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 432, 528, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block,  hog_channel=hog_channel, colorspace=colorspace))
    scale = 1.0
    boxes.extend(find_cars(img, 400, 464, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    boxes.extend(find_cars(img, 416, 480, scale, clf, X_scaler, orient=orient, pix_per_cell=pix_per_cell,
              cell_per_block=cell_per_block, hog_channel=hog_channel, colorspace=colorspace))
    
    cache.add_boxes(boxes)
        
    heatmap_img = np.zeros_like(img[:,:,0])
    for bx in cache.boxes:
        heatmap_img = add_heat(heatmap_img, bx)
    heatmap_img = apply_threshold(heatmap_img, 1 + len(cache.boxes)//2)
    
    labels = label(heatmap_img)
    
    rgb_img = color_conversion(img, 'RGB')
    draw_img, rects = draw_labeled_bboxes(rgb_img, labels)
    
    return draw_img

def video_preproc_cache(img):
    img2 = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
    return pipelineCache(img2)
In [91]:
cache = DetectionCache()

clip_test2 = VideoFileClip('test_video.mp4')
clip_test_out2 = clip_test2.fl_image(video_preproc_cache)
%time clip_test_out2.write_videofile('test_video_out_cache.mp4', audio=False)
[MoviePy] >>>> Building video test_video_out_cache.mp4
[MoviePy] Writing video test_video_out_cache.mp4
 97%|█████████▋| 38/39 [01:45<00:02,  2.77s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: test_video_out_cache.mp4 

CPU times: user 6min 46s, sys: 7 s, total: 6min 53s
Wall time: 1min 46s
In [92]:
HTML("""
<video width="640" height="480" controls>
  <source src="{0}">
</video>
""".format('test_video_out_cache.mp4'))
Out[92]:
In [ ]:
cache = DetectionCache()

clip_test2 = VideoFileClip('project_video.mp4')
clip_test_out2 = clip_test2.fl_image(video_preproc_cache)
%time clip_test_out2.write_videofile('project_video_out.mp4', audio=False)
In [ ]:
HTML("""
<video width="640" height="480" controls>
  <source src="{0}">
</video>
""".format('project_video_out.mp4'))

Performance Optimizations

In [93]:
def worker_manager(img):
    colorspace = 'YUV'
    orient = 11
    pix_per_cell = 16
    cell_per_block = 2
    hog_channel = 'ALL' 
    
    args = [
        (img, 400, 596, 3.0, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace),
        (img, 464, 660, 3.0, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace),
        (img, 400, 528, 2.0, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace),
        (img, 432, 560, 2.0, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace),
        (img, 400, 496, 1.5, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace),
        (img, 432, 528, 1.5, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace),
        (img, 400, 464, 1.0, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace),
        (img, 416, 480, 1.0, clf, X_scaler, orient, pix_per_cell, cell_per_block, hog_channel, colorspace)
    ]
    
    tmp = thread_pool.map(workerPipeline, args)
    boxes = list(itertools.chain.from_iterable(tmp))
    
    cache.add_boxes(boxes)
        
    heatmap_img = np.zeros_like(img[:,:,0])
    for bx in cache.boxes:
        heatmap_img = add_heat(heatmap_img, bx)
    heatmap_img = apply_threshold(heatmap_img, 1 + len(cache.boxes)//2)
    
    labels = label(heatmap_img)
    
    rgb_img = color_conversion(img, 'RGB')
    draw_img, rects = draw_labeled_bboxes(rgb_img, labels)
    
    return draw_img

def workerPipeline(args):
    return find_cars(*args)

def video_preproc_cache_mp(img):
    img2 = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
    return worker_manager(img2)
In [95]:
cache = DetectionCache()

clip_test2 = VideoFileClip('test_video.mp4')
clip_test_out2 = clip_test2.fl_image(video_preproc_cache_mp)
%time clip_test_out2.write_videofile('test_video_out_cache_mp.mp4', audio=False)
[MoviePy] >>>> Building video test_video_out_cache_mp.mp4
[MoviePy] Writing video test_video_out_cache_mp.mp4
 97%|█████████▋| 38/39 [01:35<00:02,  2.50s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: test_video_out_cache_mp.mp4 

CPU times: user 1min 17s, sys: 32.3 s, total: 1min 49s
Wall time: 1min 36s
In [96]:
HTML("""
<video width="640" height="480" controls>
  <source src="{0}">
</video>
""".format('test_video_out_cache_mp.mp4'))
Out[96]:
In [97]:
cache = DetectionCache()

clip_test2 = VideoFileClip('project_video.mp4')
clip_test_out2 = clip_test2.fl_image(video_preproc_cache)
%time clip_test_out2.write_videofile('project_video_out_mp.mp4', audio=False)
[MoviePy] >>>> Building video project_video_out_mp.mp4
[MoviePy] Writing video project_video_out_mp.mp4
100%|█████████▉| 1260/1261 [54:19<00:02,  2.59s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: project_video_out_mp.mp4 

CPU times: user 3h 39min 4s, sys: 3min 29s, total: 3h 42min 34s
Wall time: 54min 20s
In [98]:
HTML("""
<video width="640" height="480" controls>
  <source src="{0}">
</video>
""".format('project_video_out_mp.mp4'))
Out[98]: